#!/usr/bin/env python3
#<span style="font-size:14px;"># -*- coding: utf-8 -*-
import os
import sys
import shutil
import threading
import linecache
from time import ctime, sleep
from multiprocessing import cpu_count
import pdb
import codecs
#import exceptions
import re
import queue
#pdb.set_trace()

'''
http://blog.csdn.net/daiyinger/article/details/48289575
'''


log_mutex = threading.Lock()
list_mutex = threading.Lock()
stdout_mutex = threading.Lock()




def single_command(in_entry, key_len):
    #used for kaldi scp file generation
    if len(in_entry) == 0:
        return

    part_dir_len = key_len # this can be set by author
    entry_dir = os.path.dirname(in_entry)
    dir_units = [] #[  for i in entry_dir.split(os.path.sep) if len(i) > 0]
    unit_num = len(dir_units)
    ii = 0
    key_id = r""
    while ii < part_dir_len:
        key_id += dir_units[unit_num - ii - 1] + r"_"
        ii += 1

    out_entry = key_id + "\t" + in_entry
    return out_entry


def parse_kaldi_special_scpline(scp_line):
    if len(scp_line) == 0:
        return None
    line_units = re.split(r"\s+", scp_line)
    if len(line_units) == 2:
        return line_units[1] # real dir
    else:
        i = 0
        this_element = line_units[i]
        while i < len(line_units):
            if os.path.exists(this_element):
                return this_element
            i += 1
            this_element = line_units[i]

    return None


def command_execuator(entries, log_fp, thread_index, start, end):
    if end <= start or log_fp is None or len(entries) == 0 or thread_index < 0:
        sys.stdout.write("Condition maybe somewhat error !\n")
        return
    entry_start = start
    succ_num = 0
    while entry_start <= end:
        curr_task = entries[entry_start]
        curr_task = curr_task.strip("\n\r")
        task_destination = os.path.dirname(curr_task)
        if not os.path.exists(task_destination):
            os.makedirs(task_destination, mode=0o777)

        cmd = "unzip -f -o -d  %s -d %s" % (task_destination, curr_task)
        if stdout_mutex.acquire():
            print ("Thread : " + str(thread_index) + ":" + cmd + '\n')
            stdout_mutex.release()
        if os.system(cmd) != 0:
            if log_mutex.acquire():
                log_fp.write("Thread : " + str(thread_index) + ":" + curr_task + "\n")
                log_mutex.release()
            exit(1)
        succ_num += 1
        entry_start += 1
        
#    print "Thread-%s:%s" % (thread_index, succ_num)
    if log_mutex.acquire():
        log_fp.write("Thread : " + str(thread_index) + " : in : " + str(end-start+1) + " out : " + str(succ_num) + "\n")
    log_mutex.release()


def linux_cp_execuator(entries, log_fp, thread_index, start, end, queue):
    if end <= start or log_fp is None or len(entries) == 0 or thread_index < 0:
        sys.stdout.write("Condition maybe somewhat error !\n")
        return
    entry_start = start
    succ_num = 0
    while entry_start <= end:
        curr_task = entries[entry_start]
        curr_task = curr_task.strip("\n\r")
        #curr_task = re.split(r'[ \t]+', curr_task, 1)[1]
        print("RAW TASK : {}\n".format(curr_task))
        key, dir = re.split(r"\s+", curr_task, maxsplit=1)
        if not os.path.exists(dir):
            print('Parse dir failed, continue !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n')
            continue
        else:
            print('>< >< >< task is {}\n'.format(curr_task))
        
        queue.put(curr_task)

        succ_num += 1
        entry_start += 1

    print("Thread-{}:{}".format(thread_index, succ_num))
    if log_mutex.acquire():
        log_fp.write(
            "Thread : " + str(thread_index) + " : in : " + str(end - start + 1) + " out : " + str(succ_num) + "\n")
    log_mutex.release()



def InitCommander(cmd_ini = "cmd.ini"):
    real_command = ''
    try:
        fc = codecs.open(cmd_ini, mode='rt')
        for line in fc.readlines():
            line = line.strip()
            if re.match(r'^#', line):
                continue
            real_command = line
    except:
        sys.stdout.write('Init error !\n')
        exit(-1)

    return real_command



def scp_filter(scp_lines, id_dict):
    filtered_scp = []
    for line in scp_lines:
        id = re.split(r"\s+", line.strip())[0]
        if id in id_dict:
            filtered_scp.append(line)

    return filtered_scp


if __name__ == "__main__":
    #
#    example_dir = "1121-135824-0019 flac -c -d -s /NEWAI/Share/hys_share//LibriSpeech/train-clean-360/1121/135824/1121-135824-0019.flac |"
#    line = parse_kaldi_special_scpline(example_dir)
    kaldi_scp = r""
    dst_dir = r''
    filter_scp = ""
    if len(sys.argv) > 1:
        kaldi_scp = str(sys.argv[1])
        dst_dir = str(sys.argv[2])
        if len(sys.argv) > 2:
            filter_scp = str(sys.argv[3])

    else:
        sys.stdout.write(r"No enough parameter here, system will exit !\n")
        exit(-1)

    if not os.path.exists(dst_dir):
        os.mkdir(dst_dir)

    entries = linecache.getlines(filename=kaldi_scp)
    filter_ids = {}
    total_entry_num = len(entries)
    if filter_scp != "":
        for scp_line in linecache.getlines(filter_scp):
            id = re.split(r"\s+", scp_line.strip())[0]
            filter_ids[id] = 1
        orig_size = len(entries)
        entries = scp_filter(entries, filter_ids)
        total_entry_num = len(entries)
        print('{} reduce from {} to {}\n'.format(kaldi_scp, orig_size, total_entry_num))


    log_fp = open("command_run_log.txt", 'w')
    total_core_num = cpu_count()
    task_num_one_thread = int(total_entry_num/total_core_num) + 1
    t = ['None'] * total_core_num
    exist_entries = queue.Queue()
    for thread_index in range(0, total_core_num-1):
        entry_start = thread_index * task_num_one_thread
        entry_end = (thread_index + 1) * task_num_one_thread - 1
        if entry_end >= total_entry_num:
            entry_end = total_entry_num

        t[thread_index] = threading.Thread(target=linux_cp_execuator, args=(entries, log_fp, thread_index, entry_start, entry_end, exist_entries))
        # 启动
    for thread_i in range(0, total_core_num-1):
        t[thread_i].start()

    for index_t in range(0, total_core_num - 1):
        t[index_t].join()

    with codecs.open(filename="exists_file.scp", mode='w') as f:
        while not exist_entries.empty():
            f.write('{}\n'.format(exist_entries.get()))
    log_fp.close()
